# purpose of this program: 
# (1) output a set of documents which contribute to the top10 results of the tail 1K queries extracted from the 100K query log
# (2) output the set of compressed files needed to decompress and compress for our little experiments

from __future__ import division
from operator import itemgetter, attrgetter

import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np

print "Program Begins..."

def step1():
    print "function begins..."
    # key: docID
    # value: trecID
    docIDTrecIDDict = {}
    inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_MappingTableForGov2Dataset_with_Xdoc_values_added_sorted_by_XdocValueUsingGoodTurningDividedByNumOfPostingsForEachDocument_sortedByTrecID"
    inputFileHandler = open(inputFileName1,"r")
    lineCounter = 0
    
    currentLine = inputFileHandler.readline()
    lineCounter += 1
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentTrecID = currentLineElements[0]
        currentDocID = currentLineElements[1]
        if currentDocID not in docIDTrecIDDict:
            docIDTrecIDDict[currentDocID] = currentTrecID
        else:
            print "critical error"
            exit(1)
        
        if lineCounter % 1000000 == 0:
            print "lineCounter:",lineCounter
        
        currentLine = inputFileHandler.readline()
        lineCounter += 1
    inputFileHandler.close()
    print "docIDTrecIDDict['0']:",docIDTrecIDDict['0']
    print "len(docIDTrecIDDict):",len(docIDTrecIDDict)
    
    
    outputingDocIDAndTrecIDDict = {}
    inputFileName2 = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/tail5KResultsWithRelatedPieceNum_sortedByQID"
    inputFileHandler = open(inputFileName2,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentDocID = lineElements[1]
        if currentDocID not in outputingDocIDAndTrecIDDict:
            outputingDocIDAndTrecIDDict[currentDocID] = docIDTrecIDDict[currentDocID]
        else:
            pass
    inputFileHandler.close()
    print "len(outputingDocIDAndTrecIDDict):",len(outputingDocIDAndTrecIDDict)
    
    outputingDocIDList = []
    outputingDocIDList = outputingDocIDAndTrecIDDict.keys() 
    outputingDocIDList.sort(cmp=None, key=None, reverse=False)
    
    outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/setOfTOP10DocumentsFor1KQueries"
    outputFileHandler = open(outputFileName,"w")
    for docIDInStringFormat in outputingDocIDList:
        outputFileHandler.write(docIDInStringFormat + " " + outputingDocIDAndTrecIDDict[docIDInStringFormat] + " " + "TOP10" + "\n")
        docIDInIntFormat = int(docIDInStringFormat)
        nextDressedDocIDInIntFormat = docIDInIntFormat + 1
        nextDressedDocIDInStringFormat = str(nextDressedDocIDInIntFormat)
        if nextDressedDocIDInStringFormat not in outputingDocIDList:
            outputFileHandler.write(nextDressedDocIDInStringFormat + " " + docIDTrecIDDict[nextDressedDocIDInStringFormat] + " " + "NOTTOP10_1" + "\n")
        else:
            # try the gap with 2
            nextDressedDocIDInIntFormat += 1
            nextDressedDocIDInStringFormat = str(nextDressedDocIDInIntFormat)
            if nextDressedDocIDInStringFormat not in outputingDocIDList:
                outputFileHandler.write(nextDressedDocIDInStringFormat + " " + docIDTrecIDDict[nextDressedDocIDInStringFormat] + " " + "NOTTOP10_2" + "\n")
            else:
                print "Passed"
                
    outputFileHandler.close()
    
    print "Overall Processing Statistics:"
    print "inputFileName1:",inputFileName1
    print "inputFileName2:",inputFileName2
    print "outputFileName:",outputFileName
    print "function ends."


def step2():
    print "function begins..."
    basePath = "/data/jhe/trecdata/"
    # compressed file key
    # eg. GX000-00
    outputCompressedFilesDict = {}
    
    inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/setOfTOP10AndFakeDocumentsFor1KQueries_sortedByDocID"
    inputFileHandler = open(inputFileName,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        compressedFileKey = lineElements[1].strip().split("-")[0] + "/" + lineElements[1].strip().split("-")[1]
        if compressedFileKey not in outputCompressedFilesDict:
            outputCompressedFilesDict[compressedFileKey] = 1
        else:
            pass
    inputFileHandler.close()
    
    outputCompressedFilesList = []
    outputCompressedFilesList = outputCompressedFilesDict.keys()
    outputCompressedFilesList.sort(cmp=None, key=None, reverse=False)
    
    outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/setOfGov2CompressedFilesNeededToParse"
    outputFileHandler = open(outputFileName,"w")
    for currentOutputingCompressedFile in outputCompressedFilesList:
         completedPath = basePath + currentOutputingCompressedFile + ".gz" 
         outputFileHandler.write(completedPath + "\n")
    outputFileHandler.close()
    
    print "Overall Processing Statistics:"
    print "inputFileName: ",inputFileName
    print "outputFileName: ",outputFileName
    print "function ends."

step2()
print "Program Ends."


